#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Datetime: 2021/11/30 16:18
# @Author  : CHEN Wang
# @Site    : 
# @File    : coinmarketcap.py
# @Software: PyCharm

"""
脚本说明:
"""

import os
import json
import time
import datetime
import requests
import pandas as pd
from quant_researcher.quant.project_tool.localize import DATA_DIR
from quant_researcher.quant.project_tool.time_tool import get_today, date_shifter, str_to_timestamp
from quant_researcher.quant.project_tool.wrapper_tools.common_wrappers import deco_retry
from quant_researcher.quant.project_tool.celebrity import union_filter

api_key = 'b952a5c5-dbd9-4671-8a69-bb5949aa8190'


@deco_retry(retry=5, retry_sleep=15)
def get_recent7_ohlcvm_via_http(asset='bitcoin', **kwargs):
    """
    爬取coinmarketcap网站上的ohlcv数据，只能获取最近7日数据，历史数据需要从网上粘贴下来 [coinmarketcap 已经做了调整，无法使用了]

    :param str asset: 资产类别，默认为'bitcoin'， 必须输入完整资产代码，如bitcoin, ethereum
    :param kwargs: 关键字参数
    :return:
    """

    try:
        header = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
        url = f'https://coinmarketcap.com/currencies/{asset}/historical-data/'
        a = requests.get(url, headers=header, timeout=30)
        b = a.text.split('"historicalData":{"data":')
        if a.status_code == 200 and len(b) == 1:
            print(f'指标{asset}的最新数据获取不到，请之后再获取')
            return None
        c = b[1].split('"quotes":')
        d = c[1].split('},"status":')
        res = json.loads(d[0])
        res_list = []
        for item in res:
            quote = item.pop('quote')
            res_list.append(quote)
        df = pd.DataFrame(res_list)

        df.rename(columns={'marketCap': 'market_cap'}, inplace=True)
        df['amount'] = df['volume']
        df['volume'] = df['amount'] * 4 / (df['open'] + df['high'] + df['low'] + df['close'])
        df['date'] = df['timestamp'].str[:10]
        df.drop('timestamp', axis=1, inplace=True)
        df.set_index('date', inplace=True)
        print(f'指标{asset}最新7日OHLCVM数据获取成功')
    except Exception as err:
        if 'list index out of range' in str(err):
            print(f'指标{asset}的名字有误，找不到对应的链接  或者 {asset}的最新数据确实读不出来')
            raise err
        else:
            raise err

    return df


@deco_retry(retry=500, retry_sleep=15)
def get_asset_ohlcvm_via_http(id=None, asset=None, **kwargs):
    """
    爬取coinmarketcap网站上的该资产的全部ohlcvm数据

    :param id:
    Returns a mapping of all cryptocurrencies to unique CoinMarketCap ids.
    Per our Best Practices we recommend utilizing CMC ID instead of cryptocurrency symbols to securely identify cryptocurrencies with our other endpoints
    and in your own application logic.
    Each cryptocurrency returned includes typical identifiers such as name, symbol, and token_address for flexible mapping to id.
    :param str asset: 资产类别，默认为'bitcoin'， 必须输入完整资产代码，如bitcoin, ethereum
    :param kwargs: 关键字参数
        - range: 数据期限长度，默认为‘ALL’，支持'1D', '7D', '1M', '3M', '1Y', 'YTD', 'ALL'; 返回的数据频率取决于该资产全部数据的多少
    :return:
    """

    if id is None and asset is not None:
        file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
        file_name = os.path.join(file_path, f'all_coin_list_coinmarketcap')
        all_coin_list_df = pd.read_excel(f'{file_name}.xlsx', index_col='slug')
        id = all_coin_list_df.loc[asset, 'id']
        # coins = all_coin_list_df[all_coin_list_df['id'] == id].index[0]
    elif id is None and asset is None:
        raise ValueError('id 和 asset 必须指定一个')

    print(f'开始获取id={id}的所有OHLCVM数据')
    range = kwargs.pop('range', 'ALL')

    try:
        header = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
        url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/detail/chart?id={id}&range={range}'
        a = requests.get(url, headers=header, timeout=30)
        res = json.loads(a.text)
        if 'data' not in res.keys():
            print(f'id={id}的数据无法获取, 提示{res["status"]}')
            return None
        elif a.status_code == 200 and len(res['data']) == 0:
            print(f'id={id}的数据为空无法获取')
            return None
        res_list = []
        for item in res['data']['points']:
            res_list.append([item] + res['data']['points'][item]['v'])
        df = pd.DataFrame(res_list, columns=['time_stamp', 'close', 'amount', 'market_cap', 'price_in_btc', 'circulating_supply'])
        df['circulating_supply'] = df['circulating_supply'] / df['price_in_btc']  # 取出来的circulating_supply需要换算下
        df['volume'] = df['amount'] / (df['close'])
        df['end_date'] = pd.to_datetime(df['time_stamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert('+0000')
        df['end_date'] = df['end_date'].dt.strftime('%Y-%m-%d')
        df['datetime'] = pd.to_datetime(df['time_stamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert('+0000')
        df['datetime'] = df['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
        df.sort_values(by=['datetime'], inplace=True)

        # 有些数据取出来不是日频数据，需要进行整合
        temp_df_1 = df.groupby('end_date')[['time_stamp', 'close', 'market_cap', 'price_in_btc', 'circulating_supply', 'datetime']].last()
        temp_df_2 = df.groupby('end_date')[['amount', 'volume']].last()  # 因为取到的amount 和 volume 是过去24h的交易额，因此不需要加总
        new_df = pd.concat([temp_df_1, temp_df_2], axis=1)
        today = get_today(marker='with_n_dash')
        new_df = new_df[new_df.index != today]  # 今天的数据不完整，剔除掉

        print(f'id={id}的所有OHLCVM数据获取成功')

    except Exception as err:
        raise err

    return new_df


@deco_retry(retry=50, retry_sleep=15)
def get_total_marketcap_amount_via_http(exclude_btc=False, **kwargs):
    """
    爬取coinmarketcap网站上的所有币的市值，与总的交易金额

    :param exclude_btc: 是否包含BTC， 默认是包含的
    :param kwargs: 关键字参数
        - str period: '1Y', '2Y', '5Y', 'ALL'
    :return:
    """
    period = kwargs.pop('period', 'ALL')
    try:
        today = get_today(marker='with_n_dash')
        end = int(time.time())
        date_pair_list = []
        if period != 'ALL':
            years = int(period[0])  # 获取近多少年数据
            start_date = date_shifter(before=today, step='years', how_many=-years)
            date_pair_list.append([end, start_date])
        else:
            start_date = date_shifter(before=today, step='years', how_many=-5)  # 默认取近5年的数据，注意数据取太长会报错, start_date的最小取值2013-04-28
            while start_date > '2013-04-28':
                date_pair_list.append([end, start_date])
                end = str_to_timestamp(start_date)
                start_date = date_shifter(before=start_date, step='years', how_many=-5)
            date_pair_list.append([end, '2013-04-28'])

        header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}

        df_list = []
        for date_pair in date_pair_list:
            if exclude_btc:
                # url = f'https://web-api.coinmarketcap.com/v1.1/global-metrics/quotes/historical?format=chart_altcoin&interval=1d&time_end={date_pair[0]}&time_start={date_pair[1]}'  # 读取的数据有问题
                url = f'https://api.coinmarketcap.com/data-api/v3/global-metrics/quotes/historical?format=chart_altcoin&interval=1d&timeEnd={date_pair[0]}&timeStart={date_pair[1]}'
            else:
                # url = f'https://web-api.coinmarketcap.com/v1.1/global-metrics/quotes/historical?format=chart&interval=1d&time_end={date_pair[0]}&time_start={date_pair[1]}'  # 读取的数据有问题
                url = f'https://api.coinmarketcap.com/data-api/v3/global-metrics/quotes/historical?format=chart&interval=1d&timeEnd={date_pair[0]}&timeStart={date_pair[1]}'
            a = requests.get(url, headers=header, timeout=30)
            res = json.loads(a.text)
            if 'data' not in res.keys():
                print(f'数据无法获取, 提示{res["status"]["error_message"]}')
                return None
            elif a.status_code == 200 and len(res['data']) == 0:
                print(f'数据为空无法获取')
                return None
            res_list = []
            for item in res['data']['quotes']:
                item_quote = item.pop('quote')
                item.update(item_quote[0])
                res_list.append(item)
            df = pd.DataFrame(res_list)

            df = df[['timestamp', 'totalMarketCap', 'totalVolume24H', 'totalVolume24HReported', 'btcDominance']]
            df.rename(columns={'totalMarketCap': 'market_cap', 'totalVolume24H': 'amount', 'totalVolume24HReported': 'amount_reported'}, inplace=True)
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df['timestamp'] = df['timestamp'].apply(lambda x: x - datetime.timedelta(seconds=1))  # 时间记录的为0点，但是其实是统计的前一天的数据； 统一减去1秒，调整到昨天
            df['end_date'] = df['timestamp'].dt.strftime('%Y-%m-%d')
            df['datetime'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
            df.drop(['timestamp'], axis=1, inplace=True)
            df.set_index('end_date', inplace=True)
            df.sort_values(by=['end_date'], inplace=True)
            df_list.append(df)

        all_df = pd.concat(df_list, axis=0)
        if '2022-05-12' not in all_df.index:  #
            all_df.loc['2022-05-12', 'market_cap'] = 1225934966070.32
            all_df.loc['2022-05-12', 'amount'] = 197051431678.51
            all_df.loc['2022-05-12', 'amount_reported'] = 500887098639.66
            all_df.loc['2022-05-12', 'btcDominance'] = 44.46
            all_df.loc['2022-05-12', 'datetime'] = '2022-05-12 23:59:59'
        if '2023-01-04' not in all_df.index:  #
            all_df.loc['2023-01-04', 'market_cap'] = 820040249248.98
            all_df.loc['2023-01-04', 'amount'] = 32579218979.2
            all_df.loc['2023-01-04', 'amount_reported'] = 126276002454.92
            all_df.loc['2023-01-04', 'btcDominance'] = 39.5279
            all_df.loc['2023-01-04', 'datetime'] = '2023-01-04 23:59:59'
        # 2021-02-26， 2022-02-23数据异常，需要手动修改
        all_df.loc['2021-02-26', 'amount'] = 154578653169.09
        all_df.loc['2021-02-26', 'amount_reported'] = 399981671502.26
        all_df.loc['2022-02-23', 'amount'] = 82904025618.43

        all_df.sort_index(inplace=True)
        all_df = all_df[~all_df.index.duplicated(keep='first')]  # 去重
        all_df = all_df[all_df.index != today]  # 今天的数据不完整，剔除掉
        print(f'coinmarketcap网站上的所有币的市值，与总的交易金额获取成功')

    except Exception as err:
        raise err

    return all_df


@deco_retry(retry=50, retry_sleep=15)
def get_marketpair_amount_via_http(asset='bitcoin', **kwargs):
    """
    爬取coinmarketcap网站上的某个币在不同交易所的不同交易对的交易额

    :param asset: 是否包含BTC， 默认是包含的
    :param kwargs: 关键字参数
        - str sort: 按照什么排序， ‘cmc_rank’, 'volume'
    :return:
    """

    try:
        today = get_today(marker='with_n_dash')
        header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
        sort = kwargs.pop('sort', 'cmc_rank')
        if sort == 'cmc_rank':
            url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/market-pairs/latest?slug={asset}&start=1&limit=1000&category=spot&sort=cmc_rank_advanced'
        elif sort == 'volume':
            url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/market-pairs/latest?slug={asset}&start=1&limit=1000&category=spot&sort=volume_24h_strict'
        else:
            NotImplementedError

        a = requests.get(url, headers=header, timeout=30)
        res = json.loads(a.text)
        if 'data' not in res.keys():
            print(f'数据无法获取, 提示{res["status"]["error_message"]}')
            return None
        elif a.status_code == 200 and len(res['data']) == 0:
            print(f'数据为空无法获取')
            return None

        df = pd.DataFrame(res['data']['marketPairs'])
        df['end_date'] = today

        print(f'{asset}相关现货交易对数据获取成功')

    except Exception as err:
        raise err

    return df


@deco_retry(retry=5, retry_sleep=15)
def get_all_coinlist_via_http(status='inactive,active,untracked', method='http', tag='all', **kwargs):
    """
    获取coinmarketcap网站上目前支持的所有coins

    参考：https://coinmarketcap.com/api/documentation/v1/#operation/getV1CryptocurrencyMap

    :param status: "active" Only active cryptocurrencies are returned by default.
                   Pass inactive to get a list of cryptocurrencies that are no longer active.
                   Pass untracked to get a list of cryptocurrencies that are listed but do not yet meet methodology requirements to have tracked markets available.
                   You may pass one or more comma-separated values.
    :param str method: 数据获取的方法， 支持'http' 网页爬虫, 'api'官方api, 通过api的话需要指定api_key
    :param str tag: 某类币，支持'all', 'stablecoin', 'asset-backed-stablecoin', 'defi', 'defi-2', 'a16z-portfolio', 'binance-launchpad', 'memes', 'sports', 'web3',
                            'decentralized-exchange', 'pow', 等等，所有支持的可以通过以下函数计算得到的all_df的tags列所支持的所有选项
    :param kwargs: 关键字参数
    :return:
        输出的列包含 id, name, symbol, slug, rank, is_active, first_histotrical_data, platform, status
    """

    try:
        if method == 'http':
            header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
        elif method == 'api':
            header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
                      'X-CMC_PRO_API_KEY': api_key}
        else:
            raise NotImplementedError

        all_df_list = []
        flag = True
        i = 1
        start = 1
        page_size = 5000
        while flag and (start < 50000):
            print(f'开始进行获取第{i}页数据')
            if method == 'http':
                # 下面这个url只能获取active的
                # url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listing?start=1&limit=20000&sortBy=market_cap&sortType=desc&convert=USD&' \
                #       f'cryptoType=all&tagType=all&audited=false&' \
                #       f'aux=tags,ath,atl,high24h,low24h,num_market_pairs,cmc_rank,date_added,max_supply,circulating_supply,total_supply,volume_7d,volume_30d,self_reported_circulating_supply,self_reported_market_cap'
                # 下面这个url可以获取inactive,active,untracked
                url = f'https://api.coinmarketcap.com/data-api/v3/map/all?listing_status={status}&' \
                      f'exchangeAux=is_active,status&' \
                      f'cryptoAux=is_active,status,platform,first_historical_data,last_historical_data&' \
                      f'start={start}&limit={page_size}'
                a = requests.get(url, headers=header, timeout=30)
                res = json.loads(a.text)
                df = pd.DataFrame(res['data']['cryptoCurrencyMap'])
            elif method == 'api':
                url = f'https://pro-api.coinmarketcap.com/v1/cryptocurrency/map'
                params = {"listing_status": status,
                          "start": start,
                          "limit": page_size,
                          "sort": "id",
                          'aux': "platform,first_historical_data,last_historical_data,is_active,status",
                          }
                a = requests.get(url, params=params, headers=header, timeout=30)
                res = json.loads(a.text)
                df = pd.DataFrame(res['data'])
            else:
                raise NotImplementedError

            all_df_list.append(df)
            if df.shape[0] < page_size:  # 数据已经全部获取，无需再获取
                flag = False
            else:
                start = start + page_size
                i = i + 1

        all_df = pd.concat(all_df_list, axis=0, ignore_index=True)

        # 获取每个币的标签信息
        url = f'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/listing?start=1&limit=20000&sortBy=market_cap&sortType=desc&convert=USD&' \
              f'cryptoType=all&tagType=all&audited=false&' \
              f'aux=tags'
        a = requests.get(url, headers=header, timeout=30)
        res = json.loads(a.text)
        df = pd.DataFrame(res['data']['cryptoCurrencyList'])

        # 币的信息和标签信息合并
        all_df = all_df.merge(df[['id', 'tags']], how='left', on=['id'])

        # 所有支持的tags
        all_tags_list = []
        index_num_list = []
        for i, this_tags in enumerate(all_df['tags']):
            if this_tags == []:
                pass
            elif str(this_tags) == 'nan':
                pass
            else:
                all_tags_list.append(this_tags)
                if tag == 'all':
                    index_num_list = list(all_df.index)
                else:
                    if tag.lower() in this_tags:
                        index_num_list.append(i)

        all_tags_list = union_filter(all_tags_list, how_to_deal='or')
        all_tags_list.sort()
        all_tags = pd.Series(all_tags_list)

        all_df = all_df.iloc[index_num_list, :]
        all_df.set_index('id', inplace=True)
        all_df.sort_index(inplace=True)
        print(f'coinlist获取成功')

    except Exception as err:
        raise err

    return all_df, all_tags


if __name__ == '__main__':
    # 测试get_recent7_ohlcvm_via_http
    # get_recent7_ohlcvm_via_http(asset='bitcoin')

    # 测试get_asset_ohlcvm_via_http
    # data = get_asset_ohlcvm_via_http(id=18358)
    # data = get_asset_ohlcvm_via_http(asset='bitcoin', range='1Y')
    # file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
    # file_name = os.path.join(file_path, f'bitcoin')
    # data.to_excel(f'{file_name}.xlsx')

    # data = get_marketpair_amount_via_http(asset='ethereum', sort='volume')

    # 测试get_all_coinlist_via_http
    data = get_all_coinlist_via_http(status='inactive,active,untracked', method='http', tag='all')
    file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    file_name = os.path.join(file_path, f'all_coin_list_coinmarketcap')
    data[0].to_excel(f'{file_name}.xlsx')

    data = get_all_coinlist_via_http(status='inactive,active,untracked', method='http', tag='stablecoin')
    file_path = os.path.join(DATA_DIR, 'all_history_ohlcvm_coinmarketcap')
    file_name = os.path.join(file_path, f'all_stablecoin_list_coinmarketcap')
    data[0].to_excel(f'{file_name}.xlsx')

    # 测试 get_total_marketcap_amount_via_http
    data = get_total_marketcap_amount_via_http(exclude_btc=False)
    file_path = os.path.join(DATA_DIR, r'all_history_ohlcvm_coinmarketcap')
    file_name = os.path.join(file_path, f'total_marketcap_amount')
    data.to_excel(f'{file_name}.xlsx')
